#!/usr/bin/env python
#encoding=utf8

"""
@描述：用一个python程序执行几个文件，模拟mapper-->reduce
	   1.这次使用进称交互的形式完成，顺序执行
@日期：2015年02月03日 星期二

@作者：hongxingfan
"""

import sys
import os
from subprocess import *

Mapper = "" # 虽然这里定义多余，但是看起来明显;表示全局变量
Reducer = "" # 虽然这里定义多余，但是看起来明显;表示全局变量
NUM = 0
FILES = list() #文件列表
KV = dict()
# 参数配置
def getArgv(argv):
	i = 1
	while(i < len(argv)):
		if(argv[i] == "-mapper"):
			i += 1
			global Mapper #全局变量，保存python程序
			Mapper = os.path.abspath(argv[i])
		elif(argv[i] == "-file"):
			i += 1
			FILES.append(os.path.abspath(argv[i]))
		elif(argv[i] == "-num"):
			i += 1
			global NUM
			NUM = int(argv[i]) # reduce个数
		elif(argv[i] == "-reducer"):
			i += 1
			global Reducer
			Reducer = os.path.abspath(argv[i]) # reduce个数
		else:
			print("""
				python xxx.py -mapper yyy.py -reducer yyy.py -file f1.txt -file f2.txt -file f3.txt ... -num 2
			""")
		i += 1


# 把files传给子进程，然后按照hash(key)%NUM划分到KV中
def ViaSubProcessToMap():
	py = "/usr/bin/python " + Mapper
	for file in FILES:
		p = Popen(py + " < " + file,stdin=PIPE,stdout=PIPE,stderr=PIPE,shell=True)
		while(True):
			line = p.stdout.readline()
			line = line.strip()
			if(line == "" or p.poll() != None):
				break
			cols = line.split("\t")
			key = cols[0]
			value = cols[1]
			rd = hash(key) % NUM
			if(rd in KV):
				KV[rd].append(key + "\t" + value)
			else:
				KV[rd] = [key + "\t" + value]

# Reducer过程,从KV中读取数据，聚合运算
def ExecReducer():
	py = "/usr/bin/python " + Reducer
	for (k,v) in KV.items():
		p = Popen(py,stdin=PIPE,stdout=PIPE,stderr=PIPE,shell=True)
		for vv in v:
			#print(str(k) + "\t" + vv)
			p.stdin.write(vv + "\n")
			p.stdin.flush()
		p.stdin.write("\n")
		p.stdin.flush()
		while(True):
			line = p.stdout.readline()
			if(not line):
				break
			line = line.strip()
			if(k not in KVR):
				KVR[k] = [line]
			else:
				KVR[k].append(line)
# 以上的函数不能满足需求，stdin/stdout中必须给一个数据立马获取一个输出
def ExecReducer1():
	for(k,v) in KV.items():
		KVR = dict()
		for(vv) in v:
			cols = vv.split("\t")
			key = cols[0]
			value = int(cols[1])
			if(key in KVR):
				KVR[key] += value
			else:
				KVR[key] = value
		ll = list()
		for(k1,v1) in KVR.items():
			ll.append(k1 + "\t" + str(v1))
		KV[k] = ll
if __name__ == "__main__":
	getArgv(sys.argv)
	ViaSubProcessToMap()
	#for(k,v) in KV.items():
	#	for (vv) in v:
	#		print("%d\t%s" %(k,vv))
	#sys.exit(0)
	#ExecReducer()
	ExecReducer1()

	for(k,v) in KV.items():
		for vv in v:
			print("%d\t%s" %(k,vv))
